from pymongo import MongoClient
from bs4 import BeautifulSoup
from pprint import pprint
import requests
import datetime
import json
import time
import re

server_client = MongoClient('127.0.0.1', 27017)

server_db = server_client['knx_posts_db']
offical_posts_coll = server_db['offical_posts_coll']


class POST():
    def __init__(self):
        self.url = "http://zhaopin.gedu.org/schools/"
        self.params = {

        }
        self.data = {

        }
        self.headers = {

        }
        self.scrapy()

    def scrapy(self):
        r = requests.get('http://zhaopin.gedu.org/campus/')
        soup = BeautifulSoup(r.text)

        for city in soup.select('.fenxiao a'):
            url = 'http://zhaopin.gedu.org/' + city.get('href')
            r = requests.get(url)
            soup = BeautifulSoup(r.text)

            for jd in soup.select('.zw1'):
                date = jd.find(class_ = 'rq').text
                name = jd.find('h3').get_text().split('：')[1]
                edu = jd.find(text = re.compile('学历要求：')).parent.parent.get_text().split('：')[1]
                location = jd.find(text = re.compile('工作地点：')).parent.parent.get_text().split('：')[1]
                description = jd.find(text = re.compile('工作职责：')).parent.parent.find_next_sibling().get_text() + jd.find(text = re.compile('职位要求：')).parent.parent.find_next_sibling().get_text()

                item = {
                    "url": url,
                    'edu': edu,
                    'exp': [],
                    'name': name,
                    'date': date,
                    'lang': '',
                    'place': '',
                    'major': '',
                    'count': -1,
                    'salary': [],
                    'toSchool': True,
                    'welfare': [],
                    'funType': '',
                    'company': '环球雅思培训学校',
                    'location': location,
                    'industry': '教育',
                    'keywords': [],
                    'platform': 'offical',
                    'searchKeyword': '',
                    'description': description,
                    'subIndustry': '',
                    'stime': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
                }

                result = offical_posts_coll.replace_one({'company': '环球雅思培训学校', 'name': name, 'location': location}, item, True)

                print(item['company'], item['name'])


POST()
